import requests
from bs4 import BeautifulSoup, Tag
from urllib.parse import urljoin
import json
import re


# TODO 设置session
def setup_session():
    session = requests.Session()
    session.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    })
    return session


# TODO 存储内容到json文件
def save_file(link_metadata, filename="link_metadata.json"):
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(link_metadata, f, ensure_ascii=False, indent=2)

def save_txt_file(content, filename="content.txt"):
    with open(filename, "w", encoding="utf-8") as f:
        f.write("\n".join(content))


# TODO 获取html内容
def get_response(session, url, timeout=1000):
    try:
        response = session.get(url, timeout=timeout)
        if response.status_code != 200:
            print(f"Failed to fetch {url}. Status code: {response.status_code}")
            return None
        return response
    except Exception as e:
        print(f"Error fetching {url}: {str(e)}")
        return None


# TODO 递归提取带换行的结构化文本
def get_structured_text(element, indent=0):
    lines = []

    # 预处理：移除不需要的标签
    for tag in element.find_all(['script', 'style', 'link', 'meta']):
        tag.decompose()

    # 遍历所有子元素
    for child in element.children:
        # 处理文本节点
        if isinstance(child, str):
            stripped = child.strip()
            if stripped:
                lines.append(stripped)
        # 处理标签节点
        elif isinstance(child, Tag):
            tag_name = child.name
            # 块级元素：前后添加换行
            if tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div', 'ul', 'ol', 'li']:
                child_text = get_structured_text(child, indent + (1 if tag_name == 'li' else 0))
                if child_text.strip():
                    # 添加前缀缩进（针对列表项）
                    prefix = '  ' * indent if tag_name == 'li' else ''
                    lines.append(f"\n{prefix}{child_text}\n")
            # 行内元素：直接拼接
            elif tag_name in ['span', 'a', 'strong', 'em']:
                lines.append(get_structured_text(child, indent))
            # 特殊换行标签
            elif tag_name == 'br':
                lines.append('\n')

    # 合并处理结果
    full_text = ''.join(lines)
    # 清理多余空行
    return re.sub(r'\n{3,}', '\n\n', full_text).strip()
